{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Sample for Submarine Experiment SDK\n",
    "\n",
    "The notebook shows how to use Submarine Experiment SDK to create, get, list, log, delete Submarine Experiment."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import print_function\n",
    "import submarine\n",
    "from submarine.client.models.environment_spec import EnvironmentSpec\n",
    "from submarine.client.models.experiment_spec import ExperimentSpec\n",
    "from submarine.client.models.experiment_task_spec import ExperimentTaskSpec\n",
    "from submarine.client.models.experiment_meta import ExperimentMeta\n",
    "from submarine.client.models.code_spec import CodeSpec"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create Submarine Client"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# Host is \"<SUBMARINE_SERVER_DNS_NAME>:<SUBMARINE_SERVER_PORT>\" by default\n",
    "# If you don't wanna use Jupyter notebook which generated by Submarine,\n",
    "# please set the two environment variables \"SUBMARINE_SERVER_DNS_NAME\"\n",
    "# (e.q. localhost) and \"SUBMARINE_SERVER_PORT\" (e.q. 8080)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "submarine_client = submarine.ExperimentClient()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "### Define TensorFlow experiment spec¶\n",
    "Define Submarine spec¶\n",
    "The demo only creates a PS and worker of TF experiment to run mnist sample."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "environment = EnvironmentSpec(image=\"apache/submarine:tf-dist-mnist-test-1.0\")\n",
    "experiment_meta = ExperimentMeta(\n",
    "    name=\"mnist-dist\",\n",
    "    namespace=\"default\",\n",
    "    framework=\"Tensorflow\",\n",
    "    cmd=\"python /var/tf_dist_mnist/dist_mnist.py --train_steps=100\",\n",
    "    env_vars={\"ENV1\": \"ENV1\"},\n",
    ")\n",
    "\n",
    "worker_spec = ExperimentTaskSpec(resources=\"cpu=1,memory=1024M\", replicas=1)\n",
    "ps_spec = ExperimentTaskSpec(resources=\"cpu=1,memory=1024M\", replicas=1)\n",
    "code_spec = CodeSpec(sync_mode=\"git\", url=\"https://github.com/apache/submarine.git\")\n",
    "\n",
    "experiment_spec = ExperimentSpec(\n",
    "    meta=experiment_meta,\n",
    "    environment=environment,\n",
    "    code=code_spec,\n",
    "    spec={\"Ps\": ps_spec, \"Worker\": worker_spec},\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create experiment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'experimentId': 'experiment-1640110858229-0006',\n",
       " 'uid': '0306270d-fd79-4a42-bd50-943ad7f9a3a3',\n",
       " 'status': 'Accepted',\n",
       " 'acceptedTime': '2021-12-22T02:33:56.000+08:00',\n",
       " 'createdTime': None,\n",
       " 'runningTime': None,\n",
       " 'finishedTime': None,\n",
       " 'spec': {'meta': {'experimentId': 'experiment-1640110858229-0006',\n",
       "   'name': 'mnist-dist',\n",
       "   'namespace': 'default',\n",
       "   'framework': 'Tensorflow',\n",
       "   'cmd': 'python /var/tf_dist_mnist/dist_mnist.py --train_steps=100',\n",
       "   'envVars': {'ENV1': 'ENV1'},\n",
       "   'tags': []},\n",
       "  'environment': {'name': None,\n",
       "   'dockerImage': None,\n",
       "   'kernelSpec': None,\n",
       "   'description': None,\n",
       "   'image': 'apache/submarine:tf-dist-mnist-test-1.0'},\n",
       "  'spec': {'Ps': {'replicas': 1,\n",
       "    'resources': 'cpu=1,memory=1024M',\n",
       "    'name': None,\n",
       "    'image': None,\n",
       "    'cmd': None,\n",
       "    'envVars': None,\n",
       "    'resourceMap': {'memory': '1024M', 'cpu': '1'}},\n",
       "   'Worker': {'replicas': 1,\n",
       "    'resources': 'cpu=1,memory=1024M',\n",
       "    'name': None,\n",
       "    'image': None,\n",
       "    'cmd': None,\n",
       "    'envVars': None,\n",
       "    'resourceMap': {'memory': '1024M', 'cpu': '1'}}},\n",
       "  'code': {'syncMode': 'git',\n",
       "   'url': 'https://github.com/apache/submarine.git'}}}"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "experiment = submarine_client.create_experiment(experiment_spec=experiment_spec)\n",
    "experiment"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Get the created experiment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'experimentId': 'experiment-1640110858229-0006',\n",
       " 'uid': '0306270d-fd79-4a42-bd50-943ad7f9a3a3',\n",
       " 'status': 'Created',\n",
       " 'acceptedTime': '2021-12-22T02:33:56.000+08:00',\n",
       " 'createdTime': '2021-12-22T02:33:56.000+08:00',\n",
       " 'runningTime': None,\n",
       " 'finishedTime': None,\n",
       " 'spec': {'meta': {'experimentId': 'experiment-1640110858229-0006',\n",
       "   'name': 'mnist-dist',\n",
       "   'namespace': 'default',\n",
       "   'framework': 'Tensorflow',\n",
       "   'cmd': 'python /var/tf_dist_mnist/dist_mnist.py --train_steps=100',\n",
       "   'envVars': {'ENV1': 'ENV1'},\n",
       "   'tags': []},\n",
       "  'environment': {'name': None,\n",
       "   'dockerImage': None,\n",
       "   'kernelSpec': None,\n",
       "   'description': None,\n",
       "   'image': 'apache/submarine:tf-dist-mnist-test-1.0'},\n",
       "  'spec': {'Ps': {'replicas': 1,\n",
       "    'resources': 'cpu=1,memory=1024M',\n",
       "    'name': None,\n",
       "    'image': None,\n",
       "    'cmd': None,\n",
       "    'envVars': None,\n",
       "    'resourceMap': {'memory': '1024M', 'cpu': '1'}},\n",
       "   'Worker': {'replicas': 1,\n",
       "    'resources': 'cpu=1,memory=1024M',\n",
       "    'name': None,\n",
       "    'image': None,\n",
       "    'cmd': None,\n",
       "    'envVars': None,\n",
       "    'resourceMap': {'memory': '1024M', 'cpu': '1'}}},\n",
       "  'code': {'syncMode': 'git',\n",
       "   'url': 'https://github.com/apache/submarine.git'}}}"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "id = experiment[\"experimentId\"]\n",
    "submarine_client.get_experiment(id)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### List all running experiments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "status = \"running\"\n",
    "submarine_client.list_experiments(status=status)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Wait for the experiment to finish"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python2.7/dist-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "2021-12-21 18:34:09.528066: I tensorflow/core/platform/cpu_feature_guard.cc:137] Your CPU supports instructions that this TensorFlow binary was not compiled to use: SSE4.1 SSE4.2 AVX AVX2 FMA\n",
      "2021-12-21 18:34:09.529126: I tensorflow/core/distributed_runtime/rpc/grpc_channel.cc:215] Initialize GrpcChannelCache for job ps -> {0 -> localhost:2222}\n",
      "2021-12-21 18:34:09.529164: I tensorflow/core/distributed_runtime/rpc/grpc_channel.cc:215] Initialize GrpcChannelCache for job worker -> {0 -> experiment-1640110858229-0006-worker-0.default.svc:2222}\n",
      "2021-12-21 18:34:09.529732: I tensorflow/core/distributed_runtime/rpc/grpc_server_lib.cc:324] Started server with target: grpc://localhost:2222\n"
     ]
    }
   ],
   "source": [
    "submarine_client.wait_for_finish(id)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Get specific experiment training log"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The logs of Pod experiment-1640110858229-0006-ps-0:\n",
      "\n",
      "/usr/local/lib/python2.7/dist-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "2021-12-21 18:34:09.528066: I tensorflow/core/platform/cpu_feature_guard.cc:137] Your CPU supports instructions that this TensorFlow binary was not compiled to use: SSE4.1 SSE4.2 AVX AVX2 FMA\n",
      "2021-12-21 18:34:09.529126: I tensorflow/core/distributed_runtime/rpc/grpc_channel.cc:215] Initialize GrpcChannelCache for job ps -> {0 -> localhost:2222}\n",
      "2021-12-21 18:34:09.529164: I tensorflow/core/distributed_runtime/rpc/grpc_channel.cc:215] Initialize GrpcChannelCache for job worker -> {0 -> experiment-1640110858229-0006-worker-0.default.svc:2222}\n",
      "2021-12-21 18:34:09.529732: I tensorflow/core/distributed_runtime/rpc/grpc_server_lib.cc:324] Started server with target: grpc://localhost:2222\n",
      "The logs of Pod experiment-1640110858229-0006-worker-0:\n",
      "\n",
      "/usr/local/lib/python2.7/dist-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "2021-12-21 18:34:08.093849: I tensorflow/core/platform/cpu_feature_guard.cc:137] Your CPU supports instructions that this TensorFlow binary was not compiled to use: SSE4.1 SSE4.2 AVX AVX2 FMA\n",
      "2021-12-21 18:34:08.095284: I tensorflow/core/distributed_runtime/rpc/grpc_channel.cc:215] Initialize GrpcChannelCache for job ps -> {0 -> experiment-1640110858229-0006-ps-0.default.svc:2222}\n",
      "2021-12-21 18:34:08.095334: I tensorflow/core/distributed_runtime/rpc/grpc_channel.cc:215] Initialize GrpcChannelCache for job worker -> {0 -> localhost:2222}\n",
      "2021-12-21 18:34:08.096235: I tensorflow/core/distributed_runtime/rpc/grpc_server_lib.cc:324] Started server with target: grpc://localhost:2222\n",
      "WARNING:tensorflow:From /var/tf_dist_mnist/dist_mnist.py:239: __init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please switch to tf.train.MonitoredTrainingSession\n",
      "2021-12-21 18:34:10.457283: I tensorflow/core/distributed_runtime/master_session.cc:1017] Start master session 7124f68422637964 with config: device_filters: \"/job:ps\" device_filters: \"/job:worker/task:0\" allow_soft_placement: true\n",
      "Successfully downloaded train-images-idx3-ubyte.gz 9912422 bytes.\n",
      "Extracting /tmp/mnist-data/train-images-idx3-ubyte.gz\n",
      "Successfully downloaded train-labels-idx1-ubyte.gz 28881 bytes.\n",
      "Extracting /tmp/mnist-data/train-labels-idx1-ubyte.gz\n",
      "Successfully downloaded t10k-images-idx3-ubyte.gz 1648877 bytes.\n",
      "Extracting /tmp/mnist-data/t10k-images-idx3-ubyte.gz\n",
      "Successfully downloaded t10k-labels-idx1-ubyte.gz 4542 bytes.\n",
      "Extracting /tmp/mnist-data/t10k-labels-idx1-ubyte.gz\n",
      "job name = worker\n",
      "task index = 0\n",
      "Worker 0: Initializing session...\n",
      "Worker 0: Session initialization complete.\n",
      "Training begins @ 1640111650.733849\n",
      "1640111650.969133: Worker 0: training step 1 done (global step: 0)\n",
      "1640111650.978049: Worker 0: training step 2 done (global step: 1)\n",
      "1640111650.989931: Worker 0: training step 3 done (global step: 2)\n",
      "1640111650.999159: Worker 0: training step 4 done (global step: 3)\n",
      "1640111651.008751: Worker 0: training step 5 done (global step: 4)\n",
      "1640111651.052865: Worker 0: training step 6 done (global step: 5)\n",
      "1640111651.063632: Worker 0: training step 7 done (global step: 6)\n",
      "1640111651.072713: Worker 0: training step 8 done (global step: 7)\n",
      "1640111651.082712: Worker 0: training step 9 done (global step: 8)\n",
      "1640111651.094652: Worker 0: training step 10 done (global step: 9)\n",
      "1640111651.105416: Worker 0: training step 11 done (global step: 10)\n",
      "1640111651.153330: Worker 0: training step 12 done (global step: 11)\n",
      "1640111651.163832: Worker 0: training step 13 done (global step: 12)\n",
      "1640111651.173664: Worker 0: training step 14 done (global step: 13)\n",
      "1640111651.182991: Worker 0: training step 15 done (global step: 14)\n",
      "1640111651.191109: Worker 0: training step 16 done (global step: 15)\n",
      "1640111651.199726: Worker 0: training step 17 done (global step: 16)\n",
      "1640111651.250473: Worker 0: training step 18 done (global step: 17)\n",
      "1640111651.258586: Worker 0: training step 19 done (global step: 18)\n",
      "1640111651.266307: Worker 0: training step 20 done (global step: 19)\n",
      "1640111651.275668: Worker 0: training step 21 done (global step: 20)\n",
      "1640111651.283407: Worker 0: training step 22 done (global step: 21)\n",
      "1640111651.291265: Worker 0: training step 23 done (global step: 22)\n",
      "1640111651.299381: Worker 0: training step 24 done (global step: 23)\n",
      "1640111651.305669: Worker 0: training step 25 done (global step: 24)\n",
      "1640111651.352488: Worker 0: training step 26 done (global step: 25)\n",
      "1640111651.359732: Worker 0: training step 27 done (global step: 26)\n",
      "1640111651.365766: Worker 0: training step 28 done (global step: 27)\n",
      "1640111651.372034: Worker 0: training step 29 done (global step: 28)\n",
      "1640111651.378381: Worker 0: training step 30 done (global step: 29)\n",
      "1640111651.384172: Worker 0: training step 31 done (global step: 30)\n",
      "1640111651.391027: Worker 0: training step 32 done (global step: 31)\n",
      "1640111651.404218: Worker 0: training step 33 done (global step: 32)\n",
      "1640111651.410167: Worker 0: training step 34 done (global step: 33)\n",
      "1640111651.451277: Worker 0: training step 35 done (global step: 34)\n",
      "1640111651.458168: Worker 0: training step 36 done (global step: 35)\n",
      "1640111651.496320: Worker 0: training step 37 done (global step: 36)\n",
      "1640111651.502132: Worker 0: training step 38 done (global step: 37)\n",
      "1640111651.507339: Worker 0: training step 39 done (global step: 38)\n",
      "1640111651.512701: Worker 0: training step 40 done (global step: 39)\n",
      "1640111651.517597: Worker 0: training step 41 done (global step: 40)\n",
      "1640111651.523119: Worker 0: training step 42 done (global step: 41)\n",
      "1640111651.528645: Worker 0: training step 43 done (global step: 42)\n",
      "1640111651.534247: Worker 0: training step 44 done (global step: 43)\n",
      "1640111651.550153: Worker 0: training step 45 done (global step: 44)\n",
      "1640111651.556111: Worker 0: training step 46 done (global step: 45)\n",
      "1640111651.561790: Worker 0: training step 47 done (global step: 46)\n",
      "1640111651.567773: Worker 0: training step 48 done (global step: 47)\n",
      "1640111651.573042: Worker 0: training step 49 done (global step: 48)\n",
      "1640111651.578469: Worker 0: training step 50 done (global step: 49)\n",
      "1640111651.584431: Worker 0: training step 51 done (global step: 50)\n",
      "1640111651.590252: Worker 0: training step 52 done (global step: 51)\n",
      "1640111651.595784: Worker 0: training step 53 done (global step: 52)\n",
      "1640111651.649612: Worker 0: training step 54 done (global step: 53)\n",
      "1640111651.655372: Worker 0: training step 55 done (global step: 54)\n",
      "1640111651.660762: Worker 0: training step 56 done (global step: 55)\n",
      "1640111651.666178: Worker 0: training step 57 done (global step: 56)\n",
      "1640111651.673080: Worker 0: training step 58 done (global step: 57)\n",
      "1640111651.678989: Worker 0: training step 59 done (global step: 58)\n",
      "1640111651.684430: Worker 0: training step 60 done (global step: 59)\n",
      "1640111651.690468: Worker 0: training step 61 done (global step: 60)\n",
      "1640111651.696054: Worker 0: training step 62 done (global step: 61)\n",
      "1640111651.703274: Worker 0: training step 63 done (global step: 62)\n",
      "1640111651.750652: Worker 0: training step 64 done (global step: 63)\n",
      "1640111651.756330: Worker 0: training step 65 done (global step: 64)\n",
      "1640111651.761865: Worker 0: training step 66 done (global step: 65)\n",
      "1640111651.767066: Worker 0: training step 67 done (global step: 66)\n",
      "1640111651.773966: Worker 0: training step 68 done (global step: 67)\n",
      "1640111651.790034: Worker 0: training step 69 done (global step: 68)\n",
      "1640111651.801731: Worker 0: training step 70 done (global step: 69)\n",
      "1640111651.812638: Worker 0: training step 71 done (global step: 70)\n",
      "1640111651.851214: Worker 0: training step 72 done (global step: 71)\n",
      "1640111651.860038: Worker 0: training step 73 done (global step: 72)\n",
      "1640111651.868899: Worker 0: training step 74 done (global step: 73)\n",
      "1640111651.886605: Worker 0: training step 75 done (global step: 74)\n",
      "1640111651.896981: Worker 0: training step 76 done (global step: 75)\n",
      "1640111651.907400: Worker 0: training step 77 done (global step: 76)\n",
      "1640111651.937603: Worker 0: training step 78 done (global step: 77)\n",
      "1640111651.951628: Worker 0: training step 79 done (global step: 78)\n",
      "1640111651.958359: Worker 0: training step 80 done (global step: 79)\n",
      "1640111651.966299: Worker 0: training step 81 done (global step: 80)\n",
      "1640111651.974218: Worker 0: training step 82 done (global step: 81)\n",
      "1640111651.982264: Worker 0: training step 83 done (global step: 82)\n",
      "1640111651.989423: Worker 0: training step 84 done (global step: 83)\n",
      "1640111651.997561: Worker 0: training step 85 done (global step: 84)\n",
      "1640111652.004062: Worker 0: training step 86 done (global step: 85)\n",
      "1640111652.051784: Worker 0: training step 87 done (global step: 86)\n",
      "1640111652.057463: Worker 0: training step 88 done (global step: 87)\n",
      "1640111652.065215: Worker 0: training step 89 done (global step: 88)\n",
      "1640111652.072090: Worker 0: training step 90 done (global step: 89)\n",
      "1640111652.078558: Worker 0: training step 91 done (global step: 90)\n",
      "1640111652.085314: Worker 0: training step 92 done (global step: 91)\n",
      "1640111652.091121: Worker 0: training step 93 done (global step: 92)\n",
      "1640111652.096206: Worker 0: training step 94 done (global step: 93)\n",
      "1640111652.101504: Worker 0: training step 95 done (global step: 94)\n",
      "1640111652.151558: Worker 0: training step 96 done (global step: 95)\n",
      "1640111652.158660: Worker 0: training step 97 done (global step: 96)\n",
      "1640111652.164154: Worker 0: training step 98 done (global step: 97)\n",
      "1640111652.170239: Worker 0: training step 99 done (global step: 98)\n",
      "1640111652.176086: Worker 0: training step 100 done (global step: 99)\n",
      "1640111652.181573: Worker 0: training step 101 done (global step: 100)\n",
      "Training ends @ 1640111652.181647\n",
      "Training elapsed time: 1.447798 s\n",
      "After 100 training step(s), validation cross entropy = 1294.33\n"
     ]
    }
   ],
   "source": [
    "submarine_client.get_log(id)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Delete the experiment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'experimentId': 'experiment-1640110858229-0006',\n",
       " 'uid': '0306270d-fd79-4a42-bd50-943ad7f9a3a3',\n",
       " 'status': 'Deleted',\n",
       " 'acceptedTime': None,\n",
       " 'createdTime': None,\n",
       " 'runningTime': None,\n",
       " 'finishedTime': None,\n",
       " 'spec': {'meta': {'experimentId': 'experiment-1640110858229-0006',\n",
       "   'name': 'mnist-dist',\n",
       "   'namespace': 'default',\n",
       "   'framework': 'Tensorflow',\n",
       "   'cmd': 'python /var/tf_dist_mnist/dist_mnist.py --train_steps=100',\n",
       "   'envVars': {'ENV1': 'ENV1'},\n",
       "   'tags': []},\n",
       "  'environment': {'name': None,\n",
       "   'dockerImage': None,\n",
       "   'kernelSpec': None,\n",
       "   'description': None,\n",
       "   'image': 'apache/submarine:tf-dist-mnist-test-1.0'},\n",
       "  'spec': {'Ps': {'replicas': 1,\n",
       "    'resources': 'cpu=1,memory=1024M',\n",
       "    'name': None,\n",
       "    'image': None,\n",
       "    'cmd': None,\n",
       "    'envVars': None,\n",
       "    'resourceMap': {'memory': '1024M', 'cpu': '1'}},\n",
       "   'Worker': {'replicas': 1,\n",
       "    'resources': 'cpu=1,memory=1024M',\n",
       "    'name': None,\n",
       "    'image': None,\n",
       "    'cmd': None,\n",
       "    'envVars': None,\n",
       "    'resourceMap': {'memory': '1024M', 'cpu': '1'}}},\n",
       "  'code': {'syncMode': 'git',\n",
       "   'url': 'https://github.com/apache/submarine.git'}}}"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "submarine_client.delete_experiment(id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
